# ------------------------------------------------------------------------------
# Runs tests on "leftover signal" from simple models of risk
# Author: Cassidy Shubatt <cshubatt@gmail.com>
# To run: bash 09_leftover_signal.sh
# ------------------------------------------------------------------------------

# Libraries --------------------------------------------------------------------
message("Loading libraries...")
library(here)
library(yaml)
library(optparse)
library(tidyverse)
library(glue)
library(Matrix)
library(broom)
library(data.table) # copy()
library(pROC)
library(stargazer)

temp <- here(
  "code", "06_physician_boundedness", "01_behavioral_lasso", "temp"
)
u <- modules::use(here::here("lib", "util.R"))

# Load Data --------------------------------------------------------------------
message("Loading data...")
paths <- read_yaml(here("lib", "filepaths.yml"))
overnight_lab <- ""
predictions <- readRDS(file.path(temp, "scores__lasso__stent_or_cabg_010_day.rds"))
cohort <- readRDS(glue(paths$analysis$test_cohort))

# Get Predictions --------------------------------------------------------------
message("Running regressions...")
yhat_simp <- filter(predictions, n_coef == 4)$score[[1]]
yhat_comp <- filter(predictions, n_coef == 565)$score[[1]]

reg_df <- cohort %>%
  mutate(yhat_simp = yhat_simp) %>%
  mutate(yhat_comp = yhat_comp) %>%
  mutate(yhat_additional_signal = yhat_comp - yhat_simp) %>%
  filter(!exclude)

message("Test regression: ")
test_fit <- lm(test_010_day ~ yhat_simp + yhat_additional_signal, data = reg_df)
print(summary(test_fit))

test_simple <- lm(test_010_day ~ yhat_simp, data = reg_df)

message("Yield regression: ")
stent_fit <- lm(
  stent_or_cabg_010_day ~ yhat_simp + yhat_additional_signal,
  data = filter(reg_df, test_010_day)
)
print(summary(stent_fit))
stent_simple <- lm(
  stent_or_cabg_010_day ~ yhat_simp,
  data = filter(reg_df, test_010_day)
)

# Stargazer
message("Making stargazer table...")
stargazer_tb <- stargazer(
  test_simple, test_fit, stent_simple, stent_fit,
  omit.stat = c("f", "ser", "ll", "aic", "adj.rsq"),
  no.space = TRUE
)

message("Saving...")
write(stargazer_tb, file.path(temp, "leftover_signal_regs.tex"))

message("Done.")
